{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# origin\n",
    "# TRAIN_CLASSIFY_LEARNING_RATE = 1e-4\n",
    "\n",
    "# DENSE_NET_BLOCKS = 3\n",
    "# DENSE_NET_BLOCK_LAYERS = 5\n",
    "# DENSE_NET_INITIAL_CONV_DIM = 16\n",
    "# DENSE_NET_GROWTH_RATE = DENSE_NET_INITIAL_CONV_DIM // 2\n",
    "# # called DenseNet-BC if ENABLE_BOTTLENETCK and COMPRESSION < 1 in paper\n",
    "# DENSE_NET_ENABLE_BOTTLENETCK = False\n",
    "# DENSE_NET_TRANSITION_COMPRESSION = 1.0\n",
    "# DENSE_NET_ENABLE_DROPOUT = False\n",
    "# DENSE_NET_DROPOUT = 0.5\n",
    "\n",
    "# Total params: 250,946\n",
    "# Trainable params: 248,786\n",
    "# Non-trainable params: 2,160\n",
    "# ____________________________________________________________________________________________________\n",
    "# classify train round DenseNet-17-15\n",
    "# Epoch 1/100000000\n",
    "# Epoch 00000: val_loss improved from inf to 0.48308, saving model to /output/training_logs/classify-run-DenseNet-17-15/checkpoint-00-0.4831.hdf5\n",
    "# 336s - loss: 0.4032 - acc: 0.8438 - val_loss: 0.4831 - val_acc: 0.7500\n",
    "# Epoch 2/100000000\n",
    "# Epoch 00001: val_loss did not improve\n",
    "# 247s - loss: 0.2807 - acc: 0.8500 - val_loss: 0.5616 - val_acc: 0.7500\n",
    "# Epoch 3/100000000\n",
    "# Epoch 00002: val_loss did not improve\n",
    "# 224s - loss: 0.2225 - acc: 0.9062 - val_loss: 0.6071 - val_acc: 0.7188\n",
    "# Epoch 4/100000000\n",
    "# Epoch 00003: val_loss did not improve\n",
    "# 225s - loss: 0.2288 - acc: 0.9125 - val_loss: 0.5929 - val_acc: 0.7500\n",
    "# Epoch 5/100000000\n",
    "# Epoch 00004: val_loss improved from 0.48308 to 0.48113, saving model to /output/training_logs/classify-run-DenseNet-17-15/checkpoint-04-0.4811.hdf5\n",
    "# 237s - loss: 0.2387 - acc: 0.8812 - val_loss: 0.4811 - val_acc: 0.7500\n",
    "# Epoch 6/100000000\n",
    "# Epoch 00005: val_loss improved from 0.48113 to 0.45616, saving model to /output/training_logs/classify-run-DenseNet-17-15/checkpoint-05-0.4562.hdf5\n",
    "# 210s - loss: 0.2341 - acc: 0.9000 - val_loss: 0.4562 - val_acc: 0.7812\n",
    "# Epoch 7/100000000\n",
    "# Epoch 00006: val_loss improved from 0.45616 to 0.32753, saving model to /output/training_logs/classify-run-DenseNet-17-15/checkpoint-06-0.3275.hdf5\n",
    "# 230s - loss: 0.2135 - acc: 0.8812 - val_loss: 0.3275 - val_acc: 0.7500\n",
    "# Epoch 8/100000000\n",
    "# Epoch 00007: val_loss did not improve\n",
    "# 217s - loss: 0.2137 - acc: 0.9062 - val_loss: 0.3623 - val_acc: 0.8438\n",
    "# Epoch 9/100000000\n",
    "# Epoch 00008: val_loss did not improve\n",
    "# 227s - loss: 0.3164 - acc: 0.8438 - val_loss: 0.3411 - val_acc: 0.8438\n",
    "# Epoch 10/100000000\n",
    "# Epoch 00009: val_loss did not improve\n",
    "# 253s - loss: 0.2175 - acc: 0.8812 - val_loss: 0.3398 - val_acc: 0.8750\n",
    "# Epoch 11/100000000\n",
    "# Epoch 00010: val_loss did not improve\n",
    "# 229s - loss: 0.1724 - acc: 0.9250 - val_loss: 0.4172 - val_acc: 0.7500\n",
    "# Epoch 12/100000000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TRAIN_CLASSIFY_LEARNING_RATE = 1e-5\n",
    "# DENSE_NET_ENABLE_DROPOUT = True\n",
    "# new randomized position, OK...\n",
    "\n",
    "# Total params: 250,946\n",
    "# Trainable params: 248,786\n",
    "# Non-trainable params: 2,160\n",
    "# ____________________________________________________________________________________________________\n",
    "# classify train round DenseNet-18-0\n",
    "# Epoch 1/100000000\n",
    "#  9/10 [==========================>...] - ETA: 16s - loss: 1.8258 - acc: 0.3333Epoch 00000: val_loss improved from inf to 0.81970, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-00-0.8197.hdf5\n",
    "# 10/10 [==============================] - 272s - loss: 1.7737 - acc: 0.3375 - val_loss: 0.8197 - val_acc: 0.2500\n",
    "# Epoch 2/100000000\n",
    "#  9/10 [==========================>...] - ETA: 10s - loss: 1.4111 - acc: 0.4306Epoch 00001: val_loss improved from 0.81970 to 0.64727, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-01-0.6473.hdf5\n",
    "# 10/10 [==============================] - 215s - loss: 1.4293 - acc: 0.4375 - val_loss: 0.6473 - val_acc: 0.7188\n",
    "# Epoch 3/100000000\n",
    "#  9/10 [==========================>...] - ETA: 10s - loss: 0.9344 - acc: 0.5417Epoch 00002: val_loss improved from 0.64727 to 0.51688, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-02-0.5169.hdf5\n",
    "# 10/10 [==============================] - 230s - loss: 0.8957 - acc: 0.5500 - val_loss: 0.5169 - val_acc: 0.7500\n",
    "# Epoch 4/100000000\n",
    "#  9/10 [==========================>...] - ETA: 11s - loss: 0.7690 - acc: 0.6597Epoch 00003: val_loss did not improve\n",
    "# 10/10 [==============================] - 214s - loss: 0.7218 - acc: 0.6875 - val_loss: 0.5169 - val_acc: 0.7500\n",
    "# Epoch 5/100000000\n",
    "#  9/10 [==========================>...] - ETA: 13s - loss: 0.5940 - acc: 0.7569Epoch 00004: val_loss improved from 0.51688 to 0.48428, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-04-0.4843.hdf5\n",
    "# 10/10 [==============================] - 240s - loss: 0.5911 - acc: 0.7500 - val_loss: 0.4843 - val_acc: 0.7500\n",
    "# Epoch 6/100000000\n",
    "#  9/10 [==========================>...] - ETA: 10s - loss: 0.6249 - acc: 0.7431Epoch 00005: val_loss improved from 0.48428 to 0.46407, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-05-0.4641.hdf5\n",
    "# 10/10 [==============================] - 203s - loss: 0.6309 - acc: 0.7625 - val_loss: 0.4641 - val_acc: 0.7500\n",
    "# Epoch 7/100000000\n",
    "#  9/10 [==========================>...] - ETA: 11s - loss: 0.5236 - acc: 0.7917Epoch 00006: val_loss did not improve\n",
    "# 10/10 [==============================] - 225s - loss: 0.5697 - acc: 0.7688 - val_loss: 0.5044 - val_acc: 0.7500\n",
    "# Epoch 8/100000000\n",
    "#  9/10 [==========================>...] - ETA: 12s - loss: 0.5042 - acc: 0.7847Epoch 00007: val_loss improved from 0.46407 to 0.43801, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-07-0.4380.hdf5\n",
    "# 10/10 [==============================] - 248s - loss: 0.5277 - acc: 0.7750 - val_loss: 0.4380 - val_acc: 0.7500\n",
    "# Epoch 9/100000000\n",
    "#  9/10 [==========================>...] - ETA: 12s - loss: 0.5038 - acc: 0.7778Epoch 00008: val_loss did not improve\n",
    "# 10/10 [==============================] - 235s - loss: 0.4823 - acc: 0.7937 - val_loss: 0.4392 - val_acc: 0.7500\n",
    "# Epoch 10/100000000\n",
    "#  9/10 [==========================>...] - ETA: 11s - loss: 0.5139 - acc: 0.8125Epoch 00009: val_loss improved from 0.43801 to 0.43669, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-09-0.4367.hdf5\n",
    "# 10/10 [==============================] - 223s - loss: 0.5109 - acc: 0.8063 - val_loss: 0.4367 - val_acc: 0.7500\n",
    "# Epoch 11/100000000\n",
    "#  9/10 [==========================>...] - ETA: 11s - loss: 0.5162 - acc: 0.7847Epoch 00010: val_loss improved from 0.43669 to 0.39716, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-10-0.3972.hdf5\n",
    "# 10/10 [==============================] - 227s - loss: 0.5097 - acc: 0.7875 - val_loss: 0.3972 - val_acc: 0.7500\n",
    "# Epoch 12/100000000\n",
    "#  9/10 [==========================>...] - ETA: 12s - loss: 0.5201 - acc: 0.7986Epoch 00011: val_loss did not improve\n",
    "# 10/10 [==============================] - 237s - loss: 0.5191 - acc: 0.7937 - val_loss: 0.4293 - val_acc: 0.7500\n",
    "# Epoch 13/100000000\n",
    "#  9/10 [==========================>...] - ETA: 11s - loss: 0.5646 - acc: 0.7708Epoch 00012: val_loss improved from 0.39716 to 0.29600, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-12-0.2960.hdf5\n",
    "# 10/10 [==============================] - 235s - loss: 0.6063 - acc: 0.7625 - val_loss: 0.2960 - val_acc: 0.7812\n",
    "# Epoch 14/100000000\n",
    "#  9/10 [==========================>...] - ETA: 11s - loss: 0.4012 - acc: 0.8125Epoch 00013: val_loss did not improve\n",
    "# 10/10 [==============================] - 243s - loss: 0.4475 - acc: 0.8000 - val_loss: 0.3330 - val_acc: 0.8438\n",
    "# Epoch 15/100000000\n",
    "#  1/10 [==>...........................] - ETA: 3s - loss: 0.7420 - acc: 0.6250\n",
    "# /usr/local/lib/python3.5/dist-packages/keras/callbacks.py:120: UserWarning: Method on_batch_end() is slow compared to the batch update (0.567578). Check your callbacks.\n",
    "#   % delta_t_median)\n",
    "#  9/10 [==========================>...] - ETA: 12s - loss: 0.5946 - acc: 0.7361Epoch 00014: val_loss did not improve\n",
    "# 10/10 [==============================] - 257s - loss: 0.5880 - acc: 0.7375 - val_loss: 0.3863 - val_acc: 0.7500\n",
    "# Epoch 16/100000000\n",
    "#  9/10 [==========================>...] - ETA: 13s - loss: 0.3841 - acc: 0.8681Epoch 00015: val_loss improved from 0.29600 to 0.24742, saving model to /output/training_logs/classify-run-DenseNet-18-0/checkpoint-15-0.2474.hdf5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# debug record idx msg\n",
    "# TRAIN_CLASSIFY_LEARNING_RATE = 3e-6\n",
    "# continue to explore as below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "get meta_dict from cache\n",
      "tumor record size (975, 12)\n",
      "tumor_records_train randomized\n",
      "after resample, got 2510 samples\n",
      "start classify_train\n",
      "input\n",
      "(?, 32, 32, 32, 16)\n",
      "dense block\n",
      "(?, 32, 32, 32, 16)\n",
      "(?, 32, 32, 32, 24)\n",
      "(?, 32, 32, 32, 32)\n",
      "(?, 32, 32, 32, 40)\n",
      "(?, 32, 32, 32, 48)\n",
      "(?, 32, 32, 32, 56)\n",
      "transition block\n",
      "(?, 32, 32, 32, 56)\n",
      "(?, 16, 16, 16, 56)\n",
      "dense block\n",
      "(?, 16, 16, 16, 56)\n",
      "(?, 16, 16, 16, 64)\n",
      "(?, 16, 16, 16, 72)\n",
      "(?, 16, 16, 16, 80)\n",
      "(?, 16, 16, 16, 88)\n",
      "(?, 16, 16, 16, 96)\n",
      "transition block\n",
      "(?, 16, 16, 16, 96)\n",
      "(?, 8, 8, 8, 96)\n",
      "dense block\n",
      "(?, 8, 8, 8, 96)\n",
      "(?, 8, 8, 8, 104)\n",
      "(?, 8, 8, 8, 112)\n",
      "(?, 8, 8, 8, 120)\n",
      "(?, 8, 8, 8, 128)\n",
      "(?, 8, 8, 8, 136)\n",
      "top\n",
      "(?, 136)\n",
      "(?, 2)\n",
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 32, 32, 32, 1) 0                                            \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_1 (Conv3D)                (None, 32, 32, 32, 16 448         input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNorm (None, 32, 32, 32, 16 64          conv3d_1[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "activation_1 (Activation)        (None, 32, 32, 32, 16 0           batch_normalization_1[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_2 (Conv3D)                (None, 32, 32, 32, 8) 3464        activation_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 32, 32, 32, 24 0           conv3d_1[0][0]                   \n",
      "                                                                   conv3d_2[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNorm (None, 32, 32, 32, 24 96          concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_2 (Activation)        (None, 32, 32, 32, 24 0           batch_normalization_2[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_3 (Conv3D)                (None, 32, 32, 32, 8) 5192        activation_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)      (None, 32, 32, 32, 32 0           concatenate_1[0][0]              \n",
      "                                                                   conv3d_3[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNorm (None, 32, 32, 32, 32 128         concatenate_2[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_3 (Activation)        (None, 32, 32, 32, 32 0           batch_normalization_3[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_4 (Conv3D)                (None, 32, 32, 32, 8) 6920        activation_3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)      (None, 32, 32, 32, 40 0           concatenate_2[0][0]              \n",
      "                                                                   conv3d_4[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNorm (None, 32, 32, 32, 40 160         concatenate_3[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_4 (Activation)        (None, 32, 32, 32, 40 0           batch_normalization_4[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_5 (Conv3D)                (None, 32, 32, 32, 8) 8648        activation_4[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)      (None, 32, 32, 32, 48 0           concatenate_3[0][0]              \n",
      "                                                                   conv3d_5[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNorm (None, 32, 32, 32, 48 192         concatenate_4[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_5 (Activation)        (None, 32, 32, 32, 48 0           batch_normalization_5[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_6 (Conv3D)                (None, 32, 32, 32, 8) 10376       activation_5[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)      (None, 32, 32, 32, 56 0           concatenate_4[0][0]              \n",
      "                                                                   conv3d_6[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_7 (Conv3D)                (None, 32, 32, 32, 56 3192        concatenate_5[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "average_pooling3d_1 (AveragePool (None, 16, 16, 16, 56 0           conv3d_7[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNorm (None, 16, 16, 16, 56 224         average_pooling3d_1[0][0]        \n",
      "____________________________________________________________________________________________________\n",
      "activation_6 (Activation)        (None, 16, 16, 16, 56 0           batch_normalization_6[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_8 (Conv3D)                (None, 16, 16, 16, 8) 12104       activation_6[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)      (None, 16, 16, 16, 64 0           average_pooling3d_1[0][0]        \n",
      "                                                                   conv3d_8[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNorm (None, 16, 16, 16, 64 256         concatenate_6[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_7 (Activation)        (None, 16, 16, 16, 64 0           batch_normalization_7[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_9 (Conv3D)                (None, 16, 16, 16, 8) 13832       activation_7[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_7 (Concatenate)      (None, 16, 16, 16, 72 0           concatenate_6[0][0]              \n",
      "                                                                   conv3d_9[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNorm (None, 16, 16, 16, 72 288         concatenate_7[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_8 (Activation)        (None, 16, 16, 16, 72 0           batch_normalization_8[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_10 (Conv3D)               (None, 16, 16, 16, 8) 15560       activation_8[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_8 (Concatenate)      (None, 16, 16, 16, 80 0           concatenate_7[0][0]              \n",
      "                                                                   conv3d_10[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNorm (None, 16, 16, 16, 80 320         concatenate_8[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_9 (Activation)        (None, 16, 16, 16, 80 0           batch_normalization_9[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_11 (Conv3D)               (None, 16, 16, 16, 8) 17288       activation_9[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_9 (Concatenate)      (None, 16, 16, 16, 88 0           concatenate_8[0][0]              \n",
      "                                                                   conv3d_11[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNor (None, 16, 16, 16, 88 352         concatenate_9[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "activation_10 (Activation)       (None, 16, 16, 16, 88 0           batch_normalization_10[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_12 (Conv3D)               (None, 16, 16, 16, 8) 19016       activation_10[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_10 (Concatenate)     (None, 16, 16, 16, 96 0           concatenate_9[0][0]              \n",
      "                                                                   conv3d_12[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_13 (Conv3D)               (None, 16, 16, 16, 96 9312        concatenate_10[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "average_pooling3d_2 (AveragePool (None, 8, 8, 8, 96)   0           conv3d_13[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNor (None, 8, 8, 8, 96)   384         average_pooling3d_2[0][0]        \n",
      "____________________________________________________________________________________________________\n",
      "activation_11 (Activation)       (None, 8, 8, 8, 96)   0           batch_normalization_11[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_14 (Conv3D)               (None, 8, 8, 8, 8)    20744       activation_11[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_11 (Concatenate)     (None, 8, 8, 8, 104)  0           average_pooling3d_2[0][0]        \n",
      "                                                                   conv3d_14[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNor (None, 8, 8, 8, 104)  416         concatenate_11[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_12 (Activation)       (None, 8, 8, 8, 104)  0           batch_normalization_12[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_15 (Conv3D)               (None, 8, 8, 8, 8)    22472       activation_12[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_12 (Concatenate)     (None, 8, 8, 8, 112)  0           concatenate_11[0][0]             \n",
      "                                                                   conv3d_15[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNor (None, 8, 8, 8, 112)  448         concatenate_12[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_13 (Activation)       (None, 8, 8, 8, 112)  0           batch_normalization_13[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_16 (Conv3D)               (None, 8, 8, 8, 8)    24200       activation_13[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_13 (Concatenate)     (None, 8, 8, 8, 120)  0           concatenate_12[0][0]             \n",
      "                                                                   conv3d_16[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNor (None, 8, 8, 8, 120)  480         concatenate_13[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_14 (Activation)       (None, 8, 8, 8, 120)  0           batch_normalization_14[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_17 (Conv3D)               (None, 8, 8, 8, 8)    25928       activation_14[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_14 (Concatenate)     (None, 8, 8, 8, 128)  0           concatenate_13[0][0]             \n",
      "                                                                   conv3d_17[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNor (None, 8, 8, 8, 128)  512         concatenate_14[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_15 (Activation)       (None, 8, 8, 8, 128)  0           batch_normalization_15[0][0]     \n",
      "____________________________________________________________________________________________________\n",
      "conv3d_18 (Conv3D)               (None, 8, 8, 8, 8)    27656       activation_15[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_15 (Concatenate)     (None, 8, 8, 8, 136)  0           concatenate_14[0][0]             \n",
      "                                                                   conv3d_18[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "global_average_pooling3d_1 (Glob (None, 136)           0           concatenate_15[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 136)           0           global_average_pooling3d_1[0][0] \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 2)             274         dropout_1[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 250,946\n",
      "Trainable params: 248,786\n",
      "Non-trainable params: 2,160\n",
      "____________________________________________________________________________________________________\n",
      "classify train round DenseNet-19-31\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100000000record idx :0, from train:True. records shape:(2510, 12)\n",
      "\n",
      "record idx :1, from train:True. records shape:(2510, 12)\n",
      "record idx :2, from train:True. records shape:(2510, 12)\n",
      "record idx :3, from train:True. records shape:(2510, 12)\n",
      "record idx :4, from train:True. records shape:(2510, 12)\n",
      "record idx :5, from train:True. records shape:(2510, 12)\n",
      "record idx :6, from train:True. records shape:(2510, 12)\n",
      "record idx :7, from train:True. records shape:(2510, 12)\n",
      " 1/10 [==>...........................] - ETA: 176s - loss: 0.8798 - acc: 0.5625record idx :8, from train:True. records shape:(2510, 12)\n",
      "record idx :9, from train:True. records shape:(2510, 12)\n",
      " 2/10 [=====>........................] - ETA: 164s - loss: 0.7161 - acc: 0.6875record idx :10, from train:True. records shape:(2510, 12)\n",
      "record idx :11, from train:True. records shape:(2510, 12)\n",
      "record idx :12, from train:True. records shape:(2510, 12)\n",
      "record idx :13, from train:True. records shape:(2510, 12)\n",
      " 3/10 [========>.....................] - ETA: 137s - loss: 0.5988 - acc: 0.7292record idx :14, from train:True. records shape:(2510, 12)\n",
      "record idx :15, from train:True. records shape:(2510, 12)\n",
      "record idx :16, from train:True. records shape:(2510, 12)\n",
      "record idx :17, from train:True. records shape:(2510, 12)\n",
      " 4/10 [===========>..................] - ETA: 117s - loss: 0.4823 - acc: 0.7812record idx :18, from train:True. records shape:(2510, 12)\n",
      "record idx :19, from train:True. records shape:(2510, 12)\n",
      "record idx :20, from train:True. records shape:(2510, 12)\n",
      "record idx :21, from train:True. records shape:(2510, 12)\n",
      "record idx :22, from train:True. records shape:(2510, 12)\n",
      " 5/10 [==============>...............] - ETA: 88s - loss: 0.4929 - acc: 0.7875 record idx :23, from train:True. records shape:(2510, 12)\n",
      "record idx :24, from train:True. records shape:(2510, 12)\n",
      "record idx :25, from train:True. records shape:(2510, 12)\n",
      " 6/10 [=================>............] - ETA: 73s - loss: 0.4493 - acc: 0.8125record idx :26, from train:True. records shape:(2510, 12)\n",
      "record idx :27, from train:True. records shape:(2510, 12)\n",
      "record idx :28, from train:True. records shape:(2510, 12)\n",
      "record idx :29, from train:True. records shape:(2510, 12)\n",
      " 7/10 [====================>.........] - ETA: 53s - loss: 0.4208 - acc: 0.8214record idx :30, from train:True. records shape:(2510, 12)\n",
      "record idx :31, from train:True. records shape:(2510, 12)\n",
      "record idx :32, from train:True. records shape:(2510, 12)\n",
      "record idx :33, from train:True. records shape:(2510, 1\n",
      " 8/10 [=======================>......] - ETA: 34s - loss: 0.4181 - acc: 0.8203record idx :34, from train:True. records shape:(2510, 12)\n",
      "record idx :35, from train:True. records shape:(2510, 12)\n",
      "record idx :36, from train:True. records shape:(2510, 12)\n",
      "record idx :37, from train:True. records shape:(2510, 12)\n",
      " 9/10 [==========================>...] - ETA: 16s - loss: 0.4798 - acc: 0.7986record idx :38, from train:True. records shape:(2510, 12)\n",
      "record idx :39, from train:True. records shape:(2510, 12)\n",
      "record idx :40, from train:True. records shape:(2510, 12)\n",
      "record idx :41, from train:True. records shape:(2510, 12)\n",
      "record idx :0, from train:False. records shape:(293, 12)\n",
      "record idx :42, from train:True. records shape:(2510, 12)\n",
      "record idx :1, from train:False. records shape:(293, 12)\n",
      "record idx :43, from train:True. records shape:(2510, 12)\n",
      "record idx :2, from train:False. records shape:(293, 12)\n",
      "record idx :3, from train:False. records shape:(293, 12)\n",
      "record idx :44, from train:True. records shape:(2510, 12)\n",
      "record idx :45, from train:True. records shape:(2510, 12)\n",
      "record idx :4, from train:False. records shape:(293, 12)\n",
      "record idx :46, from train:True. records shape:(2510, 12)\n",
      "record idx :5, from train:False. records shape:(293, 12)\n",
      "record idx :47, from train:True. records shape:(2510, 12)\n",
      "record idx :6, from train:False. records shape:(293, 12)\n",
      "record idx :7, from train:False. records shape:(293, 12)\n",
      "record idx :48, from train:True. records shape:(2510, 12)\n",
      "record idx :49, from train:True. records shape:(2510, 12)\n",
      "record idx :8, from train:False. records shape:(293, 12)\n",
      "record idx :50, from train:True. records shape:(2510, 12)\n",
      "record idx :9, from train:False. records shape:(293, 12)\n",
      "record idx :51, from train:True. records shape:(2510, 12)\n",
      "record idx :10, from train:False. records shape:(293, 12)\n",
      "record idx :11, from train:False. records shape:(293, 12)\n",
      "record idx :52, from train:True. records shape:(2510, 12)\n",
      "record idx :53, from train:True. records shape:(2510, 12)\n",
      "Epoch 00000: val_loss improved from inf to 0.40894, saving model to /output/training_logs/classify-run-DenseNet-19-31/checkpoint-00-0.4089.hdf5\n",
      "record idx :54, from train:True. records shape:(2510, 12)\n",
      "record idx :55, from train:True. records shape:(2510, 12)\n",
      "10/10 [==============================] - 279s - loss: 0.5002 - acc: 0.7812 - val_loss: 0.4089 - val_acc: 0.7500\n",
      "Epoch 2/100000000\n",
      " 1/10 [==>...........................] - ETA: 8s - loss: 0.2292 - acc: 0.9375record idx :56, from train:True. records shape:(2510, 12)\n",
      " 2/10 [=====>........................] - ETA: 8s - loss: 0.3119 - acc: 0.9062record idx :57, from train:True. records shape:(2510, 12)\n",
      " 3/10 [========>.....................] - ETA: 5s - loss: 0.3057 - acc: 0.8750record idx :58, from train:True. records shape:(2510, 12)\n",
      " 4/10 [===========>..................] - ETA: 4s - loss: 0.4063 - acc: 0.8594record idx :59, from train:True. records shape:(2510, 12)\n",
      "record idx :60, from train:True. records shape:(2510, 12)\n",
      "record idx :61, from train:True. records shape:(2510, 12)\n",
      "record idx :62, from train:True. records shape:(2510, 12)\n",
      " 5/10 [==============>...............] - ETA: 12s - loss: 0.3548 - acc: 0.8750record idx :63, from train:True. records shape:(2510, 12)\n",
      "record idx :64, from train:True. records shape:(2510, 12)\n",
      "record idx :65, from train:True. records shape:(2510, 12)\n",
      "record idx :66, from train:True. records shape:(2510, 12)\n",
      " 6/10 [=================>............] - ETA: 14s - loss: 0.3814 - acc: 0.8542record idx :67, from train:True. records shape:(2510, 12)\n",
      "record idx :68, from train:True. records shape:(2510, 12)\n",
      "record idx :69, from train:True. records shape:(2510, 12)\n",
      "record idx :70, from train:True. records shape:(2510, 12)\n",
      " 7/10 [====================>.........] - ETA: 17s - loss: 0.3611 - acc: 0.8571record idx :71, from train:True. records shape:(2510, 12)\n",
      "record idx :72, from train:True. records shape:(2510, 12)\n",
      "record idx :73, from train:True. records shape:(2510, 12)\n",
      " 8/10 [=======================>......] - ETA: 15s - loss: 0.3728 - acc: 0.8438record idx :74, from train:True. records shape:(2510, 12)\n",
      "record idx :75, from train:True. records shape:(2510, 12)\n",
      "record idx :76, from train:True. records shape:(2510, 12)\n",
      "record idx :77, from train:True. records shape:(2510, 12)\n",
      "record idx :78, from train:True. records shape:(2510, 12)\n",
      "record idx :79, from train:True. records shape:(2510, 12)\n",
      " 9/10 [==========================>...] - ETA: 8s - loss: 0.3661 - acc: 0.8403 record idx :80, from train:True. records shape:(2510, 12)\n",
      "record idx :81, from train:True. records shape:(2510, 12)\n",
      "record idx :12, from train:False. records shape:(293, 12)\n",
      "record idx :82, from train:True. records shape:(2510, 12)\n",
      "record idx :13, from train:False. records shape:(293, 12)\n",
      "record idx :83, from train:True. records shape:(2510, 12)\n",
      "record idx :14, from train:False. records shape:(293, 12)\n",
      "record idx :15, from train:False. records shape:(293, 12)\n",
      "record idx :84, from train:True. records shape:(2510, 12)\n",
      "record idx :85, from train:True. records shape:(2510, 12)\n",
      "record idx :16, from train:False. records shape:(293, 12)\n",
      "record idx :86, from train:True. records shape:(2510, 12)\n",
      "record idx :17, from train:False. records shape:(293, 12)\n",
      "record idx :87, from train:True. records shape:(2510, 12)\n",
      "record idx :18, from train:False. records shape:(293, 12)\n",
      "record idx :19, from train:False. records shape:(293, 12)\n",
      "record idx :88, from train:True. records shape:(2510, 12)\n",
      "record idx :89, from train:True. records shape:(2510, 12)\n",
      "record idx :90, from train:True. records shape:(2510, 12)\n",
      "record idx :20, from train:False. records shape:(293, 12)\n",
      "record idx :91, from train:True. records shape:(2510, 12)\n",
      "record idx :21, from train:False. records shape:(293, 12)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "record idx :22, from train:False. records shape:(293, 12)\n",
      "record idx :23, from train:False. records shape:(293, 12)\n",
      "record idx :92, from train:True. records shape:(2510, 12)\n",
      "record idx :93, from train:True. records shape:(2510, 12)\n",
      "record idx :94, from train:True. records shape:(2510, 12)\n",
      "record idx :95, from train:True. records shape:(2510, 12)\n",
      "Epoch 00001: val_loss improved from 0.40894 to 0.35681, saving model to /output/training_logs/classify-run-DenseNet-19-31/checkpoint-01-0.3568.hdf5\n",
      "10/10 [==============================] - 206s - loss: 0.3609 - acc: 0.8375 - val_loss: 0.3568 - val_acc: 0.7812\n",
      "Epoch 3/100000000\n",
      " 3/10 [========>.....................] - ETA: 13s - loss: 0.5428 - acc: 0.9167record idx :96, from train:True. records shape:(2510, 12)\n",
      "record idx :97, from train:True. records shape:(2510, 12)\n",
      " 4/10 [===========>..................] - ETA: 29s - loss: 0.4832 - acc: 0.9062record idx :98, from train:True. records shape:(2510, 12)\n",
      "record idx :99, from train:True. records shape:(2510, 12)\n",
      "record idx :100, from train:True. records shape:(2510, 12)\n",
      "record idx :101, from train:True. records shape:(2510, 12)\n",
      " 5/10 [==============>...............] - ETA: 38s - loss: 0.4980 - acc: 0.9000record idx :102, from train:True. records shape:(2510, 12)\n",
      "record idx :103, from train:True. records shape:(2510, 12)\n",
      "record idx :104, from train:True. records shape:(2510, 12)\n",
      "record idx :105, from train:True. records shape:(2510, 12)\n",
      "record idx :106, from train:True. records shape:(2510, 12)\n",
      " 6/10 [=================>............] - ETA: 35s - loss: 0.4749 - acc: 0.8854record idx :107, from train:True. records shape:(2510, 12)\n",
      "record idx :108, from train:True. records shape:(2510, 12)\n",
      "record idx :109, from train:True. records shape:(2510, 12)\n",
      "record idx :110, from train:True. records shape:(2510, 12)\n",
      " 7/10 [====================>.........] - ETA: 30s - loss: 0.4829 - acc: 0.8571record idx :111, from train:True. records shape:(2510, 12)\n",
      "record idx :112, from train:True. records shape:(2510, 12)\n",
      "record idx :113, from train:True. records shape:(2510, 12)\n",
      " 8/10 [=======================>......] - ETA: 22s - loss: 0.4466 - acc: 0.8594record idx :114, from train:True. records shape:(2510, 12)\n",
      "record idx :115, from train:True. records shape:(2510, 12)\n",
      "record idx :116, from train:True. records shape:(2510, 12)\n",
      "record idx :117, from train:True. records shape:(2510, 12)\n",
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.4286 - acc: 0.8611record idx :118, from train:True. records shape:(2510, 12)\n",
      "record idx :119, from train:True. records shape:(2510, 12)\n",
      "record idx :120, from train:True. records shape:(2510, 12)\n",
      "record idx :121, from train:True. records shape:(2510, 12)\n",
      "record idx :24, from train:False. records shape:(293, 12)\n",
      "record idx :122, from train:True. records shape:(2510, 12)\n",
      "record idx :25, from train:False. records shape:(293, 12)\n",
      "record idx :123, from train:True. records shape:(2510, 12)\n",
      "record idx :26, from train:False. records shape:(293, 12)\n",
      "record idx :27, from train:False. records shape:(293, 12)\n",
      "record idx :124, from train:True. records shape:(2510, 12)\n",
      "record idx :125, from train:True. records shape:(2510, 12)\n",
      "record idx :28, from train:False. records shape:(293, 12)\n",
      "record idx :126, from train:True. records shape:(2510, 12)\n",
      "record idx :29, from train:False. records shape:(293, 12)\n",
      "record idx :127, from train:True. records shape:(2510, 12)\n",
      "record idx :30, from train:False. records shape:(293, 12)\n",
      "record idx :31, from train:False. records shape:(293, 12)\n",
      "record idx :128, from train:True. records shape:(2510, 12)\n",
      "record idx :32, from train:False. records shape:(293, 12)\n",
      "record idx :129, from train:True. records shape:(2510, 12)\n",
      "record idx :33, from train:False. records shape:(293, 12)\n",
      "record idx :130, from train:True. records shape:(2510, 12)\n",
      "record idx :34, from train:False. records shape:(293, 12)\n",
      "record idx :131, from train:True. records shape:(2510, 12)\n",
      "record idx :35, from train:False. records shape:(293, 12)\n",
      "record idx :132, from train:True. records shape:(2510, 12)\n",
      "Epoch 00002: val_loss improved from 0.35681 to 0.34936, saving model to /output/training_logs/classify-run-DenseNet-19-31/checkpoint-02-0.3494.hdf5\n",
      "record idx :133, from train:True. records shape:(2510, 1\n",
      "10/10 [==============================] - 242s - loss: 0.4116 - acc: 0.8625 - val_loss: 0.3494 - val_acc: 0.7500\n",
      "Epoch 4/100000000\n",
      "record idx :134, from train:True. records shape:(2510, 12)\n",
      " 1/10 [==>...........................] - ETA: 22s - loss: 0.2711 - acc: 0.9375record idx :135, from train:True. records shape:(2510, 12)\n",
      " 3/10 [========>.....................] - ETA: 12s - loss: 0.3218 - acc: 0.9375record idx :136, from train:True. records shape:(2510, 12)\n",
      "record idx :137, from train:True. records shape:(2510, 12)\n",
      " 4/10 [===========>..................] - ETA: 36s - loss: 0.4711 - acc: 0.8906record idx :138, from train:True. records shape:(2510, 12)\n",
      "record idx :139, from train:True. records shape:(2510, 12)\n",
      "record idx :140, from train:True. records shape:(2510, 12)\n",
      "record idx :141, from train:True. records shape:(2510, 12)\n",
      " 5/10 [==============>...............] - ETA: 38s - loss: 0.4627 - acc: 0.8750record idx :142, from train:True. records shape:(2510, 12)\n",
      "record idx :143, from train:True. records shape:(2510, 12)\n",
      "record idx :144, from train:True. records shape:(2510, 12)\n",
      "record idx :145, from train:True. records shape:(2510, 12)\n",
      "record idx :146, from train:True. records shape:(2510, 12)\n",
      " 6/10 [=================>............] - ETA: 38s - loss: 0.4396 - acc: 0.8646record idx :147, from train:True. records shape:(2510, 12)\n",
      "record idx :148, from train:True. records shape:(2510, 12)\n",
      "record idx :149, from train:True. records shape:(2510, 12)\n",
      " 7/10 [====================>.........] - ETA: 34s - loss: 0.4390 - acc: 0.8482record idx :150, from train:True. records shape:(2510, 12)\n",
      "record idx :151, from train:True. records shape:(2510, 12)\n",
      "record idx :152, from train:True. records shape:(2510, 12)\n",
      "record idx :153, from train:True. records shape:(2510, 12)\n",
      "record idx :154, from train:True. records shape:(2510, 12)\n",
      " 8/10 [=======================>......] - ETA: 24s - loss: 0.4984 - acc: 0.8125record idx :155, from train:True. records shape:(2510, 12)\n",
      "record idx :156, from train:True. records shape:(2510, 12)\n",
      "record idx :157, from train:True. records shape:(2510, 12)\n",
      "record idx :158, from train:True. records shape:(2510, 12)\n",
      " 9/10 [==========================>...] - ETA: 13s - loss: 0.4918 - acc: 0.8125record idx :159, from train:True. records shape:(2510, 12)\n",
      "record idx :160, from train:True. records shape:(2510, 12)\n",
      "record idx :161, from train:True. records shape:(2510, 12)\n",
      "record idx :36, from train:False. records shape:(293, 12)\n",
      "record idx :162, from train:True. records shape:(2510, 12)\n",
      "record idx :37, from train:False. records shape:(293, 12)\n",
      "record idx :163, from train:True. records shape:(2510, 12)\n",
      "record idx :38, from train:False. records shape:(293, 12)\n",
      "record idx :39, from train:False. records shape:(293, 12)\n",
      "record idx :164, from train:True. records shape:(2510, 12)\n",
      "record idx :165, from train:True. records shape:(2510, 12)\n",
      "record idx :40, from train:False. records shape:(293, 12)\n",
      "record idx :166, from train:True. records shape:(2510, 12)\n",
      "record idx :41, from train:False. records shape:(293, 12)\n",
      "record idx :167, from train:True. records shape:(2510, 12)\n",
      "record idx :42, from train:False. records shape:(293, 12)\n",
      "record idx :43, from train:False. records shape:(293, 12)\n",
      "record idx :168, from train:True. records shape:(2510, 12)\n",
      "record idx :44, from train:False. records shape:(293, 12)\n",
      "record idx :169, from train:True. records shape:(2510, 12)\n",
      "record idx :45, from train:False. records shape:(293, 12)\n",
      "record idx :170, from train:True. records shape:(2510, 12)\n",
      "record idx :46, from train:False. records shape:(293, 12)\n",
      "record idx :171, from train:True. records shape:(2510, 12)\n",
      "record idx :47, from train:False. records shape:(293, 12)\n",
      "record idx :172, from train:True. records shape:(2510, 12)\n",
      "record idx :173, from train:True. records shape:(2510, 12)\n",
      "record idx :174, from train:True. records shape:(2510, 12)\n",
      "Epoch 00003: val_loss improved from 0.34936 to 0.28972, saving model to /output/training_logs/classify-run-DenseNet-19-31/checkpoint-03-0.2897.hdf5\n",
      "record idx :175, from train:True. records shape:(2510, 12)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10/10 [==============================] - 242s - loss: 0.4565 - acc: 0.8187 - val_loss: 0.2897 - val_acc: 0.8438\n",
      "Epoch 5/100000000\n",
      " 3/10 [========>.....................] - ETA: 8s - loss: 0.6487 - acc: 0.7917record idx :176, from train:True. records shape:(2510, 12)\n",
      "record idx :177, from train:True. records shape:(2510, 12)\n",
      " 4/10 [===========>..................] - ETA: 27s - loss: 0.5413 - acc: 0.7969record idx :178, from train:True. records shape:(2510, 12)\n",
      "record idx :179, from train:True. records shape:(2510, 12)\n",
      "record idx :180, from train:True. records shape:(2510, 12)\n",
      "record idx :181, from train:True. records shape:(2510, 12)\n",
      "record idx :182, from train:True. records shape:(2510, 12)\n",
      "record idx :183, from train:True. records shape:(2510, 12)\n",
      " 5/10 [==============>...............] - ETA: 38s - loss: 0.5052 - acc: 0.7750record idx :184, from train:True. records shape:(2510, 12)\n",
      "record idx :185, from train:True. records shape:(2510, 12)\n",
      " 6/10 [=================>............] - ETA: 37s - loss: 0.5131 - acc: 0.7708record idx :186, from train:True. records shape:(2510, 12)\n",
      "record idx :187, from train:True. records shape:(2510, 12)\n",
      "record idx :188, from train:True. records shape:(2510, 12)\n",
      "record idx :189, from train:True. records shape:(2510, 12)\n",
      "record idx :190, from train:True. records shape:(2510, 12)\n",
      " 7/10 [====================>.........] - ETA: 33s - loss: 0.5005 - acc: 0.7679record idx :191, from train:True. records shape:(2510, 12)\n",
      "record idx :192, from train:True. records shape:(2510, 12)\n",
      "record idx :193, from train:True. records shape:(2510, 12)\n",
      " 8/10 [=======================>......] - ETA: 24s - loss: 0.4533 - acc: 0.7891record idx :194, from train:True. records shape:(2510, 12)\n",
      "record idx :195, from train:True. records shape:(2510, 12)\n",
      "record idx :196, from train:True. records shape:(2510, 12)\n",
      "record idx :197, from train:True. records shape:(2510, 1\n",
      " 9/10 [==========================>...] - ETA: 13s - loss: 0.4557 - acc: 0.7847record idx :198, from train:True. records shape:(2510, 12)\n",
      "record idx :199, from train:True. records shape:(2510, 12)\n",
      "record idx :200, from train:True. records shape:(2510, 12)\n",
      "record idx :201, from train:True. records shape:(2510, 12)\n",
      "record idx :48, from train:False. records shape:(293, 12)\n",
      "record idx :202, from train:True. records shape:(2510, 12)\n",
      "record idx :49, from train:False. records shape:(293, 12)\n",
      "record idx :203, from train:True. records shape:(2510, 12)\n",
      "record idx :50, from train:False. records shape:(293, 12)\n",
      "record idx :51, from train:False. records shape:(293, 12)\n",
      "record idx :204, from train:True. records shape:(2510, 12)\n",
      "record idx :205, from train:True. records shape:(2510, 12)\n",
      "record idx :206, from train:True. records shape:(2510, 12)\n",
      "record idx :52, from train:False. records shape:(293, 12)\n",
      "record idx :207, from train:True. records shape:(2510, 12)\n",
      "record idx :53, from train:False. records shape:(293, 12)\n",
      "record idx :54, from train:False. records shape:(293, 12)\n",
      "record idx :55, from train:False. records shape:(293, 12)\n",
      "record idx :208, from train:True. records shape:(2510, 12)\n",
      "record idx :209, from train:True. records shape:(2510, 12)\n",
      "record idx :210, from train:True. records shape:(2510, 12)\n",
      "record idx :56, from train:False. records shape:(293, 12)\n",
      "record idx :211, from train:True. records shape:(2510, 12)\n",
      "record idx :57, from train:False. records shape:(293, 12)\n",
      "record idx :58, from train:False. records shape:(293, 12)\n",
      "record idx :59, from train:False. records shape:(293, 12)\n",
      "record idx :212, from train:True. records shape:(2510, 12)\n",
      "record idx :213, from train:True. records shape:(2510, 12)\n",
      "record idx :214, from train:True. records shape:(2510, 12)\n",
      "Epoch 00004: val_loss improved from 0.28972 to 0.25587, saving model to /output/training_logs/classify-run-DenseNet-19-31/checkpoint-04-0.2559.hdf5\n",
      "record idx :215, from train:True. records shape:(2510, 12)\n",
      "10/10 [==============================] - 247s - loss: 0.4574 - acc: 0.7937 - val_loss: 0.2559 - val_acc: 0.8750\n",
      "Epoch 6/100000000\n",
      " 3/10 [========>.....................] - ETA: 10s - loss: 0.4868 - acc: 0.7292record idx :216, from train:True. records shape:(2510, 12)\n",
      "record idx :217, from train:True. records shape:(2510, 12)\n",
      "record idx :218, from train:True. records shape:(2510, 12)\n",
      " 4/10 [===========>..................] - ETA: 21s - loss: 0.4408 - acc: 0.7656record idx :219, from train:True. records shape:(2510, 12)\n",
      "record idx :220, from train:True. records shape:(2510, 12)\n",
      "record idx :221, from train:True. records shape:(2510, 12)\n",
      "record idx :222, from train:True. records shape:(2510, 12)\n",
      " 5/10 [==============>...............] - ETA: 36s - loss: 0.4816 - acc: 0.7750record idx :223, from train:True. records shape:(2510, 12)\n",
      "record idx :224, from train:True. records shape:(2510, 12)\n",
      "record idx :225, from train:True. records shape:(2510, 12)\n",
      " 6/10 [=================>............] - ETA: 33s - loss: 0.4090 - acc: 0.8125record idx :226, from train:True. records shape:(2510, 12)\n",
      "record idx :227, from train:True. records shape:(2510, 12)\n",
      "record idx :228, from train:True. records shape:(2510, 12)\n",
      "record idx :229, from train:True. records shape:(2510, 12)\n",
      " 7/10 [====================>.........] - ETA: 30s - loss: 0.3799 - acc: 0.8304record idx :230, from train:True. records shape:(2510, 12)\n",
      "record idx :231, from train:True. records shape:(2510, 12)\n",
      "record idx :232, from train:True. records shape:(2510, 12)\n",
      "record idx :233, from train:True. records shape:(2510, 12)\n",
      " 8/10 [=======================>......] - ETA: 23s - loss: 0.4011 - acc: 0.8281record idx :234, from train:True. records shape:(2510, 12)\n",
      "record idx :235, from train:True. records shape:(2510, 12)\n",
      "record idx :236, from train:True. records shape:(2510, 12)\n",
      "record idx :237, from train:True. records shape:(2510, 12)\n",
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.3890 - acc: 0.8333record idx :238, from train:True. records shape:(2510, 12)\n",
      "record idx :239, from train:True. records shape:(2510, 12)\n",
      "record idx :240, from train:True. records shape:(2510, 12)\n",
      "record idx :241, from train:True. records shape:(2510, 12)\n",
      "record idx :60, from train:False. records shape:(293, 12)\n",
      "record idx :242, from train:True. records shape:(2510, 12)\n",
      "record idx :61, from train:False. records shape:(293, 12)\n",
      "record idx :243, from train:True. records shape:(2510, 12)\n",
      "record idx :62, from train:False. records shape:(293, 12)\n",
      "record idx :63, from train:False. records shape:(293, 12)\n",
      "record idx :244, from train:True. records shape:(2510, 12)\n",
      "record idx :245, from train:True. records shape:(2510, 12)\n",
      "record idx :64, from train:False. records shape:(293, 12)\n",
      "record idx :246, from train:True. records shape:(2510, 12)\n",
      "record idx :65, from train:False. records shape:(293, 12)\n",
      "record idx :247, from train:True. records shape:(2510, 12)\n",
      "record idx :66, from train:False. records shape:(293, 12)\n",
      "record idx :67, from train:False. records shape:(293, 12)\n",
      "record idx :248, from train:True. records shape:(2510, 12)\n",
      "record idx :249, from train:True. records shape:(2510, 12)\n",
      "record idx :68, from train:False. records shape:(293, 12)\n",
      "record idx :250, from train:True. records shape:(2510, 12)\n",
      "record idx :69, from train:False. records shape:(293, 12)\n",
      "record idx :251, from train:True. records shape:(2510, 12)\n",
      "record idx :70, from train:False. records shape:(293, 12)\n",
      "record idx :71, from train:False. records shape:(293, 12)\n",
      "record idx :252, from train:True. records shape:(2510, 12)\n",
      "record idx :253, from train:True. records shape:(2510, 12)\n",
      "Epoch 00005: val_loss improved from 0.25587 to 0.22001, saving model to /output/training_logs/classify-run-DenseNet-19-31/checkpoint-05-0.2200.hdf5\n",
      "record idx :254, from train:True. records shape:(2510, 12)\n",
      "10/10 [==============================]record idx :255, from train:True. records shape:(2510, 12) - 270s - loss: 0.4086 - acc: 0.8313 - val_loss: 0.2200 - val_acc: 0.9062\n",
      "\n",
      "Epoch 7/100000000\n",
      " 3/10 [========>.....................] - ETA: 10s - loss: 0.3996 - acc: 0.8958record idx :256, from train:True. records shape:(2510, 12)\n",
      "record idx :257, from train:True. records shape:(2510, 12)\n",
      " 4/10 [===========>..................] - ETA: 32s - loss: 0.3933 - acc: 0.8750record idx :258, from train:True. records shape:(2510, 12)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "record idx :259, from train:True. records shape:(2510, 12)\n",
      "record idx :260, from train:True. records shape:(2510, 12)\n",
      "record idx :261, from train:True. records shape:(2510, 12)\n",
      " 5/10 [==============>...............] - ETA: 34s - loss: 0.3986 - acc: 0.8625record idx :262, from train:True. records shape:(2510, 12)\n",
      "record idx :263, from train:True. records shape:(2510, 12)\n",
      "record idx :264, from train:True. records shape:(2510, 12)\n",
      "record idx :265, from train:True. records shape:(2510, 12)\n",
      " 6/10 [=================>............] - ETA: 35s - loss: 0.4739 - acc: 0.8333record idx :266, from train:True. records shape:(2510, 12)\n",
      "record idx :267, from train:True. records shape:(2510, 12)\n",
      "record idx :268, from train:True. records shape:(2510, 12)\n",
      "record idx :269, from train:True. records shape:(2510, 12)\n",
      " 7/10 [====================>.........] - ETA: 34s - loss: 0.4325 - acc: 0.8482record idx :270, from train:True. records shape:(2510, 12)\n",
      "record idx :271, from train:True. records shape:(2510, 12)\n",
      "record idx :272, from train:True. records shape:(2510, 12)\n",
      "record idx :273, from train:True. records shape:(2510, 12)\n",
      " 8/10 [=======================>......] - ETA: 24s - loss: 0.4436 - acc: 0.8359record idx :274, from train:True. records shape:(2510, 12)\n",
      "record idx :275, from train:True. records shape:(2510, 12)\n",
      "record idx :276, from train:True. records shape:(2510, 12)\n",
      "record idx :277, from train:True. records shape:(2510, 12)\n",
      "record idx :278, from train:True. records shape:(2510, 12)\n",
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.4281 - acc: 0.8333record idx :279, from train:True. records shape:(2510, 12)\n",
      "record idx :280, from train:True. records shape:(2510, 12)\n",
      "record idx :281, from train:True. records shape:(2510, 12)\n",
      "record idx :282, from train:True. records shape:(2510, 12)\n",
      "record idx :72, from train:False. records shape:(293, 12)\n",
      "record idx :283, from train:True. records shape:(2510, 12)\n",
      "record idx :73, from train:False. records shape:(293, 12)\n",
      "record idx :74, from train:False. records shape:(293, 12)\n",
      "record idx :75, from train:False. records shape:(293, 12)\n",
      "record idx :284, from train:True. records shape:(2510, 12)\n",
      "record idx :285, from train:True. records shape:(2510, 12)\n",
      "record idx :286, from train:True. records shape:(2510, 12)\n",
      "record idx :76, from train:False. records shape:(293, 12)\n",
      "record idx :287, from train:True. records shape:(2510, 12)\n",
      "record idx :77, from train:False. records shape:(293, 12)\n",
      "record idx :78, from train:False. records shape:(293, 12)\n",
      "record idx :79, from train:False. records shape:(293, 12)\n",
      "record idx :288, from train:True. records shape:(2510, 12)\n",
      "record idx :289, from train:True. records shape:(2510, 12)\n",
      "record idx :290, from train:True. records shape:(2510, 12)\n",
      "record idx :80, from train:False. records shape:(293, 12)\n",
      "record idx :291, from train:True. records shape:(2510, 12)\n",
      "record idx :81, from train:False. records shape:(293, 12)\n",
      "record idx :82, from train:False. records shape:(293, 12)\n",
      "record idx :83, from train:False. records shape:(293, 12)\n",
      "record idx :292, from train:True. records shape:(2510, 12)\n",
      "record idx :293, from train:True. records shape:(2510, 12)\n",
      "record idx :294, from train:True. records shape:(2510, 12)\n",
      "Epoch 00006: val_loss did not improve\n",
      "10/10 [==============================] - 243s - loss: 0.4171 - acc: 0.8313 - val_loss: 0.2978 - val_acc: 0.8750\n",
      "Epoch 8/100000000\n",
      "record idx :295, from train:True. records shape:(2510, 12)\n",
      " 3/10 [========>.....................] - ETA: 17s - loss: 0.3994 - acc: 0.8542record idx :296, from train:True. records shape:(2510, 12)\n",
      "record idx :297, from train:True. records shape:(2510, 12)\n",
      " 4/10 [===========>..................] - ETA: 38s - loss: 0.3640 - acc: 0.8594record idx :298, from train:True. records shape:(2510, 12)\n",
      "record idx :299, from train:True. records shape:(2510, 12)\n",
      "record idx :300, from train:True. records shape:(2510, 12)\n",
      "record idx :301, from train:True. records shape:(2510, 12)\n",
      " 5/10 [==============>...............] - ETA: 45s - loss: 0.4134 - acc: 0.8500record idx :302, from train:True. records shape:(2510, 12)\n",
      "record idx :303, from train:True. records shape:(2510, 12)\n",
      "record idx :304, from train:True. records shape:(2510, 12)\n",
      "record idx :305, from train:True. records shape:(2510, 12)\n",
      " 6/10 [=================>............] - ETA: 41s - loss: 0.4516 - acc: 0.8125record idx :306, from train:True. records shape:(2510, 12)\n",
      "record idx :307, from train:True. records shape:(2510, 12)\n",
      "record idx :308, from train:True. records shape:(2510, 12)\n",
      "record idx :309, from train:True. records shape:(2510, 12)\n",
      " 7/10 [====================>.........] - ETA: 35s - loss: 0.4328 - acc: 0.8036record idx :310, from train:True. records shape:(2510, 12)\n",
      "record idx :311, from train:True. records shape:(2510, 12)\n",
      "record idx :312, from train:True. records shape:(2510, 12)\n",
      "record idx :313, from train:True. records shape:(2510, 12)\n",
      " 8/10 [=======================>......] - ETA: 24s - loss: 0.3886 - acc: 0.8281record idx :314, from train:True. records shape:(2510, 12)\n",
      "record idx :315, from train:True. records shape:(2510, 12)\n",
      "record idx :316, from train:True. records shape:(2510, 12)\n",
      "record idx :317, from train:True. records shape:(2510, 12)\n",
      "record idx :318, from train:True. records shape:(2510, 12)\n",
      " 9/10 [==========================>...] - ETA: 12s - loss: 0.3686 - acc: 0.8333record idx :319, from train:True. records shape:(2510, 12)\n",
      "record idx :320, from train:True. records shape:(2510, 12)\n",
      "record idx :321, from train:True. records shape:(2510, 12)\n",
      "record idx :84, from train:False. records shape:(293, 12)\n",
      "record idx :322, from train:True. records shape:(2510, 12)\n",
      "record idx :85, from train:False. records shape:(293, 12)\n",
      "record idx :323, from train:True. records shape:(2510, 12)\n",
      "record idx :86, from train:False. records shape:(293, 12)\n",
      "record idx :87, from train:False. records shape:(293, 12)\n",
      "record idx :324, from train:True. records shape:(2510, 12)\n",
      "record idx :325, from train:True. records shape:(2510, 12)\n",
      "record idx :88, from train:False. records shape:(293, 12)\n",
      "record idx :326, from train:True. records shape:(2510, 12)\n",
      "record idx :89, from train:False. records shape:(293, 12)\n",
      "record idx :327, from train:True. records shape:(2510, 12)\n",
      "record idx :90, from train:False. records shape:(293, 12)\n",
      "record idx :91, from train:False. records shape:(293, 12)\n",
      "record idx :328, from train:True. records shape:(2510, 12)\n",
      "record idx :92, from train:False. records shape:(293, 12)\n",
      "record idx :329, from train:True. records shape:(2510, 12)\n",
      "record idx :93, from train:False. records shape:(293, 12)\n",
      "record idx :330, from train:True. records shape:(2510, 12)\n",
      "record idx :94, from train:False. records shape:(293, 12)\n",
      "record idx :331, from train:True. records shape:(2510, 12)\n",
      "record idx :95, from train:False. records shape:(293, 12)\n",
      "record idx :332, from train:True. records shape:(2510, 12)\n",
      "Epoch 00007: val_loss did not improve\n",
      "10/10 [==============================] - 240s - loss: 0.3582 - acc: 0.8375 - val_loss: 0.2835 - val_acc: 0.8750\n",
      "Epoch 9/100000000\n",
      "record idx :333, from train:True. records shape:(2510, 12)\n",
      " 1/10 [==>...........................] - ETA: 6s - loss: 0.2719 - acc: 0.8750record idx :334, from train:True. records shape:(2510, 12)\n",
      " 2/10 [=====>........................] - ETA: 8s - loss: 0.3036 - acc: 0.8438record idx :335, from train:True. records shape:(2510, 12)\n",
      " 3/10 [========>.....................] - ETA: 6s - loss: 0.3576 - acc: 0.8333record idx :336, from train:True. records shape:(2510, 12)\n",
      "record idx :337, from train:True. records shape:(2510, 12)\n",
      " 4/10 [===========>..................] - ETA: 30s - loss: 0.3134 - acc: 0.8594record idx :338, from train:True. records shape:(2510, 12)\n",
      "record idx :339, from train:True. records shape:(2510, 12)\n",
      "record idx :340, from train:True. records shape:(2510, 12)\n",
      "record idx :341, from train:True. records shape:(2510, 12)\n",
      " 5/10 [==============>...............] - ETA: 42s - loss: 0.2874 - acc: 0.8750record idx :342, from train:True. records shape:(2510, 12)\n",
      "record idx :343, from train:True. records shape:(2510, 12)\n",
      "record idx :344, from train:True. records shape:(2510, 12)\n",
      "record idx :345, from train:True. records shape:(2510, 12)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 6/10 [=================>............] - ETA: 39s - loss: 0.3453 - acc: 0.8646record idx :346, from train:True. records shape:(2510, 12)\n",
      "record idx :347, from train:True. records shape:(2510, 12)\n",
      "record idx :348, from train:True. records shape:(2510, 12)\n",
      "record idx :349, from train:True. records shape:(2510, 12)\n",
      "record idx :350, from train:True. records shape:(2510, 12)\n",
      " 7/10 [====================>.........] - ETA: 31s - loss: 0.3509 - acc: 0.8661record idx :351, from train:True. records shape:(2510, 12)\n",
      "record idx :352, from train:True. records shape:(2510, 12)\n",
      "record idx :353, from train:True. records shape:(2510, 12)\n",
      "record idx :354, from train:True. records shape:(2510, 12)\n",
      " 8/10 [=======================>......] - ETA: 21s - loss: 0.3769 - acc: 0.8438record idx :355, from train:True. records shape:(2510, 12)\n",
      "record idx :356, from train:True. records shape:(2510, 12)\n",
      "record idx :357, from train:True. records shape:(2510, 12)\n",
      " 9/10 [==========================>...] - ETA: 11s - loss: 0.3583 - acc: 0.8542record idx :358, from train:True. records shape:(2510, 12)\n",
      "record idx :359, from train:True. records shape:(2510, 12)\n",
      "record idx :360, from train:True. records shape:(2510, 12)\n",
      "record idx :361, from train:True. records shape:(2510, 12)\n",
      "record idx :362, from train:True. records shape:(2510, 12)\n",
      "record idx :96, from train:False. records shape:(293, 12)\n",
      "record idx :363, from train:True. records shape:(2510, 12)\n",
      "record idx :97, from train:False. records shape:(293, 12)\n",
      "record idx :98, from train:False. records shape:(293, 12)\n",
      "record idx :99, from train:False. records shape:(293, 12)\n",
      "record idx :364, from train:True. records shape:(2510, 12)\n",
      "record idx :365, from train:True. records shape:(2510, 12)\n",
      "record idx :366, from train:True. records shape:(2510, 12)\n",
      "record idx :100, from train:False. records shape:(293, 12)\n",
      "record idx :367, from train:True. records shape:(2510, 12)\n",
      "record idx :101, from train:False. records shape:(293, 12)\n",
      "record idx :102, from train:False. records shape:(293, 12)\n",
      "record idx :103, from train:False. records shape:(293, 12)\n",
      "record idx :368, from train:True. records shape:(2510, 12)\n",
      "record idx :369, from train:True. records shape:(2510, 12)\n",
      "record idx :370, from train:True. records shape:(2510, 12)\n",
      "record idx :104, from train:False. records shape:(293, 12)\n",
      "record idx :371, from train:True. records shape:(2510, 12)\n",
      "record idx :105, from train:False. records shape:(293, 12)\n",
      "record idx :106, from train:False. records shape:(293, 12)\n",
      "record idx :107, from train:False. records shape:(293, 12)\n",
      "record idx :372, from train:True. records shape:(2510, 12)\n",
      "record idx :373, from train:True. records shape:(2510, 12)\n",
      "record idx :374, from train:True. records shape:(2510, 12)\n",
      "Epoch 00008: val_loss did not improve\n",
      "10/10 [==============================] - 260s - loss: 0.3602 - acc: 0.8562 - val_loss: 0.2601 - val_acc: 0.8750\n",
      "Epoch 10/100000000\n",
      "record idx :375, from train:True. records shape:(2510, 12)\n",
      " 3/10 [========>.....................] - ETA: 15s - loss: 0.2296 - acc: 0.9167"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-4e7dcbb2cb28>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtrain_classification\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mclassify_train\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mclassify_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/root/Tianchi-Medical-LungTumorDetect/train_classification.py\u001b[0m in \u001b[0;36mclassify_train\u001b[0;34m()\u001b[0m\n\u001b[1;32m     37\u001b[0m                         \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mget_classify_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTRAIN_CLASSIFY_VALID_BATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfrom_train\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTRAIN_CLASSIFY_VALID_STEPS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m                         \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTRAIN_CLASSIFY_EPOCHS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m                         callbacks=[tensorboard, checkpoint, early_stopping])\n\u001b[0m\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     86\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     87\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 88\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_support_signature\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetargspec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_q_size, workers, pickle_safe, initial_epoch)\u001b[0m\n\u001b[1;32m   1857\u001b[0m                             \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1858\u001b[0m                         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1859\u001b[0;31m                             \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwait_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1860\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1861\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerator_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'__len__'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from train_classification import classify_train\n",
    "classify_train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
